from time import sleep

import requests
from lxml import etree
import pandas as pd

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.69'
}

urls = [
    'https://www.17k.com/top/refactor/top100/01_subscribe/01_subscribe__top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/14_recommend/14_recommend_top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/16_gift/16_gift_top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/18_popularityListScore/18_popularityListScore_top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/18_popularityListScore/18_popularityListScore_finishBook_top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/06_vipclick/06_click_freeBook_top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/17_honor/17_honor_top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/10_bookshelf/10_bookshelf_top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/09_totalwords/09_totalwords_top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/12_hotcomment/12_hotcomment_top_100_pc.html',
    'https://www.17k.com/top/refactor/top100/11_frequentlyupdate/11_frequentlyupdate_top_100_pc.html'
]


def main():
    df_list = []
    for url in urls:
        html_doc = crawling(url)
        rank_title, rank_info_list = extract_data(html_doc)
        df = pd.DataFrame(rank_info_list[1:], columns=rank_info_list[0])
        df['排行榜'] = pd.DataFrame([rank_title for i in range(100)], columns=['排行榜'])
        df_list.append(df)
        sleep(1)
    save_data(df_list)


def crawling(url):
    response = requests.get(url, headers=headers)
    html = response.content.decode()
    html_doc = etree.HTML(html)
    return html_doc


def extract_data(html_doc):
    div = html_doc.xpath('//div[@class="content TABBOX"]')[0]
    rank_title = div.xpath('./div[1]/h2/text()')[0]
    rank_tr = div.xpath('./div[2]/div[1]//tr')
    rank_info_list = [i.xpath('.//td/text()|th/text()|td/a/text()|td/img/@alt') for i in rank_tr]
    return rank_title, rank_info_list


def save_data(df_list):
    df = pd.concat(df_list, ignore_index=False)
    df.to_csv('../static/data/rank_info.csv', index=False)


if __name__ == '__main__':
    main()
